import json
import re
import requests
import time

from db import *
from settings import stocks_url, lei
# from pg import cur, conn


def spiderStock(page, url):
    header = {}
    response = requests.get(url % page, header)
    reRes = re.findall(re.compile("[(](.*)[)]"), response.text)
    return json.loads(reRes[0])


def collect_stock(url):
    tmpList = []
    count = 0
    for i in range(1, 143):
        res = spiderStock(i, url)
        stockNameList = res["data"]["diff"]
        for name in stockNameList:
            tmpDict = {}
            if name["f2"] == '-':
                continue
            tmpDict["name"] = name["f14"]
            tmpDict["code"] = name["f12"]
            tmpDict["price"] = name["f20"]
            if name["f13"]:
                tmpDict["prefix"] = "SH"
            else:
                tmpDict["prefix"] = "SZ"
            if str(tmpDict["code"]).startswith("8") or str(tmpDict["code"]).startswith("4") or str(tmpDict["code"]).startswith("9"):
                tmpDict["prefix"] = "BJ"

            tmpDict["exist"] = 1
            tmpDict["symbol"] = tmpDict['prefix'].upper() + tmpDict["code"]
            if '退' in tmpDict["name"] or tmpDict["name"] in lei:
                tmpDict["exist"] = 0
            count += 1
            print(tmpDict["symbol"], name["f13"], count)

            stock_db.update_one({"symbol": tmpDict["symbol"]}, {
                             "$set": tmpDict}, upsert=True)

        time.sleep(0.2)
    return tmpList


if __name__ == "__main__":
    collect_stock(stocks_url)
